{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 将所有特征串联起来，构成RS_Train.csv\n",
    "#RS_Test.csv\n",
    "#为最后推荐系统做准备\n",
    "\n",
    "#导入需要的工具包\n",
    "from __future__ import division\n",
    "\n",
    "import pickle\n",
    "import numpy as np\n",
    "import scipy.io as sio\n",
    "import scipy.sparse as ss\n",
    "from numpy.random import random  \n",
    "from collections import defaultdict\n",
    "import scipy.spatial.distance as ssd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "class RecommonderSystem:\n",
    "    def __init__(self):\n",
    "        # 读入数据做初始化\n",
    "\n",
    "        #用户和活动新的索引(训练集和测试集共有)\n",
    "        self.userIndex = pickle.load(open(\"PE_userIndex.pkl\", 'rb'))\n",
    "        self.eventIndex = pickle.load(open(\"PE_eventIndex.pkl\", 'rb'))\n",
    "        self.n_users = len(self.userIndex)\n",
    "        self.n_items = len(self.eventIndex)\n",
    "\n",
    "        #用户-活动关系矩阵R\n",
    "        self.userEventScores = sio.mmread(\"PE_userEventScores\").todense()\n",
    "\n",
    "        #倒排表\n",
    "        ##每个用户参加的事件\n",
    "        self.itemsForUser = pickle.load(open(\"PE_eventsForUser.pkl\", 'rb'))\n",
    "        ##每个事件参加的用户\n",
    "        self.usersForItem = pickle.load(open(\"PE_usersForEvent.pkl\", 'rb'))\n",
    "\n",
    "        #基于模型的协同过滤参数初始化,训练\n",
    "        self.init_SVD()\n",
    "        self.train_SVD(trainfile = \"train.csv\")\n",
    "\n",
    "        #根据用户属性计算出的用户之间的相似度\n",
    "        self.userSimMatrix = sio.mmread(\"US_userSimMatrix\").todense()\n",
    "\n",
    "        #根据活动属性计算出的活动之间的相似度\n",
    "        self.eventPropSim = sio.mmread(\"EV_eventPropSim\").todense()\n",
    "        self.eventContSim = sio.mmread(\"EV_eventContSim\").todense()\n",
    "\n",
    "        #每个用户的朋友的数目\n",
    "        self.numFriends = sio.mmread(\"UF_numFriends\")\n",
    "        #用户的每个朋友参加活动的分数对该用户的影响\n",
    "        self.userFriends = sio.mmread(\"UF_userFriends\").todense()\n",
    "\n",
    "        #活动本身的热度\n",
    "        self.eventPopularity = sio.mmread(\"EA_eventPopularity\").todense()\n",
    "\n",
    "    def init_SVD(self, K=20):\n",
    "        #初始化模型参数（for 基于模型的协同过滤SVD_CF）\n",
    "        self.K = K  \n",
    "\n",
    "        #init parameters\n",
    "        #bias\n",
    "        self.bi = np.zeros(self.n_items)  \n",
    "        self.bu = np.zeros(self.n_users)  \n",
    "\n",
    "        #the small matrix\n",
    "        self.P = random((self.n_users,self.K))/10*(np.sqrt(self.K))\n",
    "        self.Q = random((self.K, self.n_items))/10*(np.sqrt(self.K))  \n",
    "                  \n",
    "          \n",
    "    def train_SVD(self,trainfile = 'train.csv', steps=10,gamma=0.04,Lambda=0.15):\n",
    "#         pass\n",
    "        #训练SVD模型（for 基于模型的协同过滤SVD_CF）\n",
    "        #gamma：为学习率\n",
    "        #Lambda：正则参数\n",
    "\n",
    "        print(\"SVD Train...\")\n",
    "        ftrain = open(trainfile, 'r',encoding=\"utf-8\")\n",
    "        ftrain.readline()\n",
    "        self.mu = 0.0\n",
    "        n_records = 0\n",
    "        uids = []  #每条记录的用户索引\n",
    "        i_ids = [] #每条记录的item索引\n",
    "        #用户-Item关系矩阵R（内容同userEventScores相同），临时变量，训练完了R不再需要\n",
    "        R = np.zeros((self.n_users, self.n_items))\n",
    "\n",
    "        for line in ftrain:\n",
    "            cols = line.strip().split(\",\")\n",
    "            u = self.userIndex[cols[0]]  #用户索引\n",
    "            i = self.eventIndex[cols[1]] #活动索引\n",
    "\n",
    "            uids.append(u)\n",
    "            i_ids.append(i)\n",
    "\n",
    "            R[u,i] = int(cols[4])  #interested\n",
    "            self.mu += R[u,i]\n",
    "            n_records += 1\n",
    "\n",
    "        ftrain.close()\n",
    "        self.mu /= n_records #平均得分\n",
    "\n",
    "        # 完整SVD模型训练过程\n",
    "        \n",
    "        uids = uids[:5]\n",
    "#         print(uids)\n",
    "        i_ids = i_ids[:5]\n",
    "#         print(i_ids)\n",
    "        ks = self.K #隐含因子\n",
    "        e = np.zeros((self.n_users, self.n_items)) #误差矩阵初始化\n",
    "\n",
    "        #偏置矩阵处理（用户重复问题）\n",
    "        bu = np.zeros((self.n_users, self.n_items))\n",
    "        bi = np.zeros((self.n_users, self.n_items))\n",
    "        for u in uids:\n",
    "            for i in i_ids:\n",
    "                bu[u,i] = self.bu[u]\n",
    "                bi[u,i] = self.bi[i]\n",
    "                #print(u,i,bu[u,i])\n",
    "        #迭代        \n",
    "        for step in range(steps):\n",
    "            for u in uids:\n",
    "                for i in i_ids:\n",
    "                    p_q = np.zeros((self.n_users, self.n_items)) #p,q矩阵乘积和\n",
    "                    for k in range(ks):\n",
    "                        p_q[u,i] += self.P[u,k]*self.Q[k,i]\n",
    "                    e[u,i] = R[u,i]-self.mu-bu[u,i]-bi[u,i]-p_q[u,i] #误差项\n",
    "                    #print(u,i,e[u,i])\n",
    "\n",
    "            \n",
    "            #更新/梯度下降          \n",
    "            for u in uids:\n",
    "                for i in i_ids:\n",
    "                    #bias update\n",
    "                    bu[u,i] -= gamma*(-e[u,i]+Lambda*bu[u,i]) # 用户属性更新                   \n",
    "                    bi[u,i] -= gamma*(-e[u,i]+Lambda*bi[u,i]) #事件属性更新\n",
    "                    #print(bi[u,i])\n",
    "                    #正则项更新            \n",
    "                    for k in range(ks):\n",
    "                        self.P[u,k] -= gamma*(-e[u,i]*self.Q[k,i]+Lambda*self.P[u,k]) #P矩阵更新\n",
    "                        self.Q[k,i] -= gamma*(-e[u,i]*self.P[u,k]+Lambda*self.Q[k,i]) #Q矩阵更新\n",
    "                        #print(self.P[u,k])\n",
    "         #bias还原   \n",
    "        for u in uids:\n",
    "            for i in i_ids:\n",
    "                self.bu[u] = bu[u,i]\n",
    "                self.bi[i] = bi[u,i]\n",
    "                #print(u,i,bu[u,i])\n",
    "        \n",
    "        print(\"SVD trained\")\n",
    "\n",
    "    def pred_SVD(self, uid, i_id):\n",
    "        #根据当前参数，预测用户uid对Item（i_id）的打分        \n",
    "        ans = self.mu + self.bi[i_id] + self.bu[uid] + np.dot(self.P[uid,:],self.Q[:,i_id])  \n",
    "\n",
    "        #将打分范围控制在0-1之间\n",
    "        if ans>1:  \n",
    "            return 1  \n",
    "        elif ans<0:  \n",
    "            return 0\n",
    "        return ans  \n",
    "\n",
    "    def sim_cal_UserCF(self, uid1, uid2):\n",
    "        #基于用户的协同过滤中的两个用户uid1和uid2之间的相似度（根据两个用户对item打分的相似度）\n",
    "        similarity = 0.0\n",
    "        similarity = 1-self.userSimMatrix[uid1, uid2] #从相似度矩阵中取值，文件采用.correlation计算，                                                                                  #需要用1减去此关系\n",
    "        return similarity  \n",
    "\n",
    "    def userCFReco(self, userId, eventId): \n",
    "        #基于用户的协同过滤，得到event的推荐度\n",
    "        ans = 0.0\n",
    "        sim_d = dict() #用户与其他用户相似度关系\n",
    "        sims = [] #用户与其他用户的相似度列表，用于后续排序 \n",
    "        \n",
    "        #用户索引处理\n",
    "        uid1 = self.userIndex[userId]\n",
    "        i_id = self.eventIndex[eventId]\n",
    "        \n",
    "        #相似度\n",
    "        for user2 in self.userIndex.keys():           \n",
    "            uid2 = self.userIndex[user2]\n",
    "            sim = self.sim_cal_UserCF(uid1,uid2) #调用相似度函数\n",
    "            sims.append(sim)\n",
    "            sim_d[sim] = uid2\n",
    "        \n",
    "        #对相似度排序，取排名前5的相似度进行加权平均计算\n",
    "        sims.sort(reverse = True) #对相似度进行排序\n",
    "        rank = 5 \n",
    "        sims = sims[1:6]\n",
    "        a = 0.0 #分子\n",
    "        b = 0.0\n",
    "        for r in range(rank):\n",
    "            a += sims[r]*self.userEventScores[sim_d[sims[r]],i_id] #相似度加权\n",
    "            b += sims[r] #分母，相似度求和\n",
    "        ans = a/b #基于用户的协同过滤结果\n",
    "        return ans\n",
    "\n",
    "    def sim_cal_ItemCF(self, i_id1, i_id2):\n",
    "        #计算Item i_id1和i_id2之间的相似度\n",
    "        similarity = 0.0\n",
    "        similarity = self.eventPropSim[i_id1,i_id2]\n",
    "        return similarity     \n",
    "        \n",
    "    def eventCFReco(self, userId, eventId):\n",
    "        #基于事件的协同过滤，得到event的推荐度\n",
    "        ans = 0.0 #初始化\n",
    "        sim_d = dict()\n",
    "        sims = []\n",
    "        \n",
    "        uid = self.userIndex[userId] #用户、事件的索引\n",
    "        i_id1 = self.eventIndex[eventId]\n",
    "        \n",
    "        for i in self.eventIndex.keys():\n",
    "            i_id2 = self.eventIndex[i]\n",
    "            sim = self.sim_cal_ItemCF(i_id1,i_id2)#调用事件的相似度\n",
    "            sims.append(sim) #相似度列表\n",
    "            sim_d[sim] = i_id2 #相似度索引\n",
    "            \n",
    "        sims.sort(reverse = True)#相似度排序\n",
    "        rank = 5\n",
    "        sims = sims[1:6]#切片\n",
    "        a = 0.0\n",
    "        b = 0.0\n",
    "        for r in range(rank):\n",
    "            a += sims[r]*self.userEventScores[uid,sim_d[sims[r]]] #分子相似度加权\n",
    "            b +=sims[r]#分母相似度之和\n",
    "        ans = a/b\n",
    "        return ans\n",
    "\n",
    "    def svdCFReco(self, userId, eventId):\n",
    "        #基于模型的协同过滤, SVD++/LFM\n",
    "        u = self.userIndex[userId]\n",
    "        i = self.eventIndex[eventId]\n",
    "\n",
    "        return self.pred_SVD(u,i)\n",
    "\n",
    "    def userReco(self, userId, eventId):\n",
    "        \n",
    "        #用户之间的相似度由用户本身的属性得到，计算event的推荐度\n",
    "\n",
    "        i = self.userIndex[userId]\n",
    "        j = self.eventIndex[eventId]\n",
    "\n",
    "        vs = self.userEventScores[:, j]\n",
    "        sims = self.userSimMatrix[i, :]\n",
    "\n",
    "        prod = sims * vs\n",
    "\n",
    "        try:\n",
    "            return prod[0, 0] - self.userEventScores[i, j]\n",
    "        except IndexError:\n",
    "            return 0\n",
    "\n",
    "    def eventReco(self, userId, eventId):\n",
    "        #item之间的相似度由item本身的属性得到，计算Event的推荐度        \n",
    "        i = self.userIndex[userId]\n",
    "        j = self.eventIndex[eventId]\n",
    "        js = self.userEventScores[i, :]\n",
    "        psim = self.eventPropSim[:, j]\n",
    "        csim = self.eventContSim[:, j]\n",
    "        pprod = js * psim\n",
    "        cprod = js * csim\n",
    "\n",
    "        pscore = 0\n",
    "        cscore = 0\n",
    "        try:\n",
    "            pscore = pprod[0, 0] - self.userEventScores[i, j]\n",
    "        except IndexError:\n",
    "            pass\n",
    "        try:\n",
    "            cscore = cprod[0, 0] - self.userEventScores[i, j]\n",
    "        except IndexError:\n",
    "            pass\n",
    "        return pscore, cscore\n",
    "\n",
    "    def userPop(self, userId):\n",
    "        \"\"\"\n",
    "        基于用户的朋友个数来推断用户的社交程度\n",
    "        主要的考量是如果用户的朋友非常多，可能会更倾向于参加各种社交活动\n",
    "        \"\"\"\n",
    "        if userId in self.userIndex.keys():\n",
    "            i = self.userIndex[userId]\n",
    "            try:\n",
    "                return self.numFriends[0, i]\n",
    "            except IndexError:\n",
    "                return 0\n",
    "        else:\n",
    "            return 0\n",
    "\n",
    "    def friendInfluence(self, userId):\n",
    "        \"\"\"\n",
    "        朋友对用户的影响\n",
    "        主要考虑用户所有的朋友中，有多少是非常喜欢参加各种社交活动/event的\n",
    "        用户的朋友圈如果都积极参与各种event，可能会对当前用户有一定的影响\n",
    "        \"\"\"\n",
    "        nusers = np.shape(self.userFriends)[1]\n",
    "        i = self.userIndex[userId]\n",
    "        return (self.userFriends[i, :].sum(axis=0) / nusers)[0,0]\n",
    "\n",
    "    def eventPop(self, eventId):\n",
    "        pass\n",
    "        \"\"\"\n",
    "        本活动本身的热度\n",
    "        主要是通过参与的人数来界定的\n",
    "        \"\"\"\n",
    "        i = self.eventIndex[eventId]\n",
    "        return self.eventPopularity[i, 0]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SVD Train...\n",
      "SVD trained\n"
     ]
    }
   ],
   "source": [
    "RS = RecommonderSystem() #基于模型的协同过滤，采用部分数据简单测试一下，电脑跑不动所有的"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.7900033131207649"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "RS.userCFReco(\"3044012\", \"2529072432\") #基于用户的协同过滤测试"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.0"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "RS.eventCFReco(\"3044012\", \"2529072432\") #基于事件的协同过滤测试"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def generateRSData(RS, train=True, header=True):\n",
    "    \"\"\"\n",
    "    特征重组\n",
    "    把前面user-based协同过滤 和 item-based协同过滤，以及各种热度和影响度作为特征组合在一起\n",
    "    生成新的训练数据，用于分类器分类使用\n",
    "    \"\"\"\n",
    "    fn = \"train.csv\" if train else \"test.csv\"\n",
    "    fin = open(fn, 'r',encoding=\"utf-8\")\n",
    "    fout = open(\"RS_\" + fn, 'w',encoding=\"utf-8\")\n",
    "    \n",
    "    #忽略第一行（列名字）\n",
    "    fin.readline().strip().split(\",\")\n",
    "    \n",
    "    # 创建新的标题行\n",
    "    if header:\n",
    "        ocolnames = [\"invited\", \"userCF_reco\", \"evtCF_reco\",\"svdCF_reco\",\"user_reco\", \"evt_p_reco\",\n",
    "        \"evt_c_reco\", \"user_pop\", \"frnd_infl\", \"evt_pop\"]\n",
    "        if train: #训练集需要加上用户是否对事件感兴趣\n",
    "            ocolnames.append(\"interested\")\n",
    "            ocolnames.append(\"not_interested\")\n",
    "    fout.write(\",\".join(ocolnames) + \"\\n\") #将标题行写入文件\n",
    "    ln = 0\n",
    "    for line in fin:\n",
    "        ln += 1\n",
    "        if ln%11 == 0:\n",
    "            print(\"%s:%d (userId, eventId)=(%s, %s)\" % (fn, ln, userId, eventId))\n",
    "            print(\"电脑内存太小，只重组了10行数据进行保存\")\n",
    "            break;\n",
    "\n",
    "        \n",
    "        cols = line.strip().split(\",\") #用户行信息\n",
    "        \n",
    "        #用户各个特征计算及重组        \n",
    "        userId = cols[0]  #用户ID\n",
    "        eventId = cols[1]  #事件ID\n",
    "        invited = cols[2]  #是否被邀请\n",
    "        userCF_reco = RS.userCFReco(userId, eventId) #基于用户的协同过滤\n",
    "        itemCF_reco = RS.eventCFReco(userId, eventId) #基于事件的协同过滤\n",
    "        svdCF_reco = RS.svdCFReco(userId, eventId) #基于模型的协同过滤\n",
    "        user_reco = RS.userReco(userId, eventId) #基于用户本身\n",
    "        evt_p_reco, evt_c_reco = RS.eventReco(userId, eventId) #基于事件本身\n",
    "        user_pop = RS.userPop(userId) #基于用户好友个数\n",
    "        frnd_infl = RS.friendInfluence(userId) #基于用户好友影响力\n",
    "        evt_pop = RS.eventPop(eventId) #基于活动热度\n",
    "            \n",
    "        ocols = [invited, userCF_reco, itemCF_reco, svdCF_reco,user_reco, evt_p_reco,\n",
    "        evt_c_reco, user_pop, frnd_infl, evt_pop] #用户信息重组\n",
    "\n",
    "        if train:\n",
    "            ocols.append(cols[4]) # interested\n",
    "            ocols.append(cols[5]) # not_interested #训练集补充用户对事件是否感兴趣\n",
    "        fout.write(\",\".join(map(lambda x: str(x), ocols)) + \"\\n\") #用户重组信息处理并保存        \n",
    "        #break\n",
    "    fin.close()\n",
    "    fout.close() #新的训练数据保存完毕"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "h:\\python36\\lib\\site-packages\\ipykernel_launcher.py:208: RuntimeWarning: invalid value encountered in double_scalars\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train.csv:11 (userId, eventId)=(4236494, 110357109)\n",
      "电脑内存太小，只重组了10行数据进行保存\n"
     ]
    }
   ],
   "source": [
    "generateRSData(RS, train=True, header=True) #训练集特征重组并保存"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "h:\\python36\\lib\\site-packages\\ipykernel_launcher.py:208: RuntimeWarning: invalid value encountered in double_scalars\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "test.csv:11 (userId, eventId)=(5161061, 2169802745)\n",
      "电脑内存太小，只重组了10行数据进行保存\n"
     ]
    }
   ],
   "source": [
    "generateRSData(RS, train=False, header=True) #测试集特征重组并保存"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
